Noorderpoort
np_domain <- query_db("SELECT * FROM domain WHERE name = 'noorderpoort.memorylab.app';", database = "slimstampen")
Users registered on this domain:
np_users <- query_db(paste0("SELECT id AS user_id, email FROM users WHERE domain_id = ", np_domain$id, ";"), database = "slimstampen")
Lessons on this domain:
np_lessons <- query_db(paste0("SELECT * FROM lesson WHERE domain_id = ", np_domain$id, ";"), database = "slimstampen")
Sessions from users on this domain during the pilot period:
np_sessions <- query_db(paste0("SELECT * FROM session WHERE token_id = 2 AND user_id IN (", paste(np_users$user_id, collapse = ", "), ");"), database = "ssaas")
np_sessions <- np_sessions[create_time > "2024-10-06"][create_time < "2024-11-05"]
When were these sessions?
np_sessions[, session_date := as.Date(create_time)]
np_test_dates <- data.table(test = c("Toets 1", "Toets 2"),
date = as.Date(c("2024-10-07", "2024-11-04")))
ggplot(np_sessions, aes(x = session_date)) +
geom_vline(data = np_test_dates, aes(xintercept = date), linetype = "dashed", colour = "grey50") +
geom_label(data = np_test_dates, aes(x = date, y = Inf, label = test), vjust = 1.05, hjust = .5, colour = "grey50") +
geom_histogram(binwidth = 1, fill = colours_memorylab[1]) +
labs(x = "Datum", y = "MemoryLab oefensessies per dag", title = "MemoryLab oefenactiviteit over de tijd") +
scale_x_date(date_labels = "%d/%m", date_breaks = "1 week", limits = c(as.Date("2024-10-06"), as.Date("2024-11-05"))) +
scale_y_continuous(expand = c(0, 0)) +
theme_ml() +
theme(panel.grid.major.y = element_line(colour = "grey90"))
ggsave(here("output", "memorylab_oefensessies_over_tijd.png"), width = 10, height = 5)

Most popular days:
np_sessions[, .N, by = .(session_date)][order(-N)]
Total sessions:
nrow(np_sessions)
[1] 696
Sessions per user:
np_sessions[, .N, by = user_id][order(-N)]
Total users with at least one session:
length(unique(np_sessions$user_id))
[1] 108
Which lessons did users do? Parse the session context:
np_sessions[, context_parsed := map(context, function (x) {
x <- fromJSON(x)
data.table(lesson_id = x$lessonId,
lesson_group_id = x$lessonGroupId,
title = x$title)
})]
np_sessions_parsed <- np_sessions[, rbindlist(context_parsed), by = .(session_id = id, user_id, create_time, session_date)]
Practice by lesson:
np_sessions_parsed[, .(`Keer geoefend` = .N), by = .(Les = title)][order(-`Keer geoefend`)] |> knitr::kable()
| Tafels_les 1 |
104 |
| Afronden op decimalen |
75 |
| Procenten |
69 |
| Optellen |
51 |
| Breuken verkennen |
50 |
| Afronden op hele getallen |
49 |
| Aftrekken |
45 |
| Delen |
38 |
| Getallen en cijfers |
36 |
| Tafels_les 2 |
26 |
| Vermenigvuldigen met grote getallen |
25 |
| Breuken vermenigvuldigen |
23 |
| Breuken versimpelen |
19 |
| Rekentaal - Belangrijke woorden & Afkortingen
(1) |
16 |
| Decimale getallen |
12 |
| Tafels_les 5 |
11 |
| Grootheden |
10 |
| Rekentaal - Meetkunde |
7 |
| Metriek stelsel |
7 |
| Tafels_les 3 |
6 |
| Vermenigvuldigen met kommagetallen |
5 |
| Decimale getallen + en - |
3 |
| Rekentaal - Getalbegrippen (1) |
2 |
| Tafels_les 4 |
2 |
| Rekentaal - Belangrijke woorden & Afkortingen
(3) |
2 |
| Rekentaal - Symbolen en tekens |
1 |
| Rekentaal - Belangrijke woorden & Afkortingen
(2) |
1 |
| Rekentaal - Ruimtebegrippen (1) |
1 |
We can link each practice session to one of the test topics:
np_lesson_groups <- data.table(
lesson_group_id = c(
29092,
29087,
29096,
29089,
29098,
29086,
29097,
29095,
29090,
29088,
29094,
29091),
topic = c(
"Rekentaal",
"Breuken",
"Percentage",
"Eenheden",
"Delen",
"Afronden",
"Cijfers",
"Vermenigvuldigen",
"Aftrekken & Optellen",
"Decimalen",
"Tafels",
"Rekentaal"
)
)
# Set lesson topics to the right order
np_lesson_groups[, topic := factor(topic, levels = c("Delen",
"Percentage",
"Cijfers",
"Breuken",
"Tafels",
"Decimalen",
"Aftrekken & Optellen",
"Vermenigvuldigen",
"Afronden",
"Eenheden",
"Rekentaal"))]
np_lessons <- merge(np_lessons, np_lesson_groups, by = "lesson_group_id")
np_sessions_parsed <- merge(np_sessions_parsed, np_lesson_groups, by = "lesson_group_id")
Practice by lesson topic:
np_sessions_parsed[, .(`Keer geoefend` = .N), by = .(Onderwerp = topic)][order(-`Keer geoefend`)] |> knitr::kable()
| Tafels |
149 |
| Afronden |
124 |
| Aftrekken & Optellen |
96 |
| Breuken |
92 |
| Percentage |
69 |
| Delen |
38 |
| Cijfers |
36 |
| Rekentaal |
30 |
| Vermenigvuldigen |
30 |
| Eenheden |
17 |
| Decimalen |
15 |
Bar plot of sessions per topic:
np_sessions_parsed[, .(`Keer geoefend` = .N), by = .(Onderwerp = topic)] |>
ggplot(aes(x = Onderwerp, y = `Keer geoefend`)) +
geom_col(fill = colours_memorylab[1]) +
labs(x = "Onderwerp", y = "Keer geoefend", caption = "Data from noorderpoort.memorylab.app") +
theme_ml()

Mastery credits:
np_credits <- query_db(paste0("SELECT * FROM lesson_mastered WHERE user_id IN (", paste(np_users$user_id, collapse = ", "), ");"), database = "slimstampen")
# Add lesson titles and topics
np_credits <- merge(np_credits, np_lessons[, .(lesson_id = id, title, topic)])
Credits by topic:
np_credits[, .N, by = .(topic)][order(-N)]
Responses from these user_ids:
np_responses <- query_db(paste0("SELECT * FROM response WHERE token_id = 2 AND user_id IN (", paste(np_users$user_id, collapse = ", "), ");"), database = "ssaas")
np_responses <- np_responses[create_time > "2024-10-06"][create_time < "2024-11-05"]
# Add lesson titles and topics
np_responses <- merge(np_responses, np_sessions_parsed[, .(session_id, title, topic)], by = "session_id")
Responses by user:
np_responses[, .N, by = .(user_id)][order(-N)]
Responses by topic:
np_responses[, .N, by = .(topic)][order(-N)]
ggplot(np_responses, aes(x = as.Date(create_time))) +
geom_histogram(binwidth = 1, fill = colours_memorylab[1]) +
labs(x = "Date", y = "Responses per day", caption = "Data from noorderpoort.memorylab.app") +
scale_y_continuous(expand = c(0, 0), labels = scales::number_format(big.mark = ",")) +
theme_ml() +
theme(panel.grid.major.y = element_line(colour = "grey90"))

Split by lesson topic:
ggplot(np_responses, aes(x = as.Date(create_time))) +
geom_histogram(aes(fill = topic), binwidth = 1) +
labs(x = "Date", y = "Responses per day", fill = "Onderwerp", caption = "Data from noorderpoort.memorylab.app") +
scale_y_continuous(expand = c(0, 0), labels = scales::number_format(big.mark = ",")) +
scale_fill_viridis_d() +
theme_ml() +
theme(panel.grid.major.y = element_line(colour = "grey90"))

How does study behaviour on specific topics relate to test
performance?
We want to see whether studying a specific topic is related to an
increase in test performance. Studying behaviour can be summarised in
several ways: time spent, number of sessions, number of questions
answered, number of credits achieved.
np_session_stats <- np_responses[, .(
n_responses = .N,
duration = max(presentation_start_time) + presentation_duration[which.max(presentation_start_time)] - min(presentation_start_time),
accuracy = mean(correct)
), by = .(user_id, topic, session_id)]
np_practice_stats <- np_session_stats[, .(
n_sessions = .N,
n_responses = sum(n_responses),
duration = sum(duration),
accuracy = mean(accuracy)
), by = .(user_id, topic)]
Load test scores per topic:
np_test_scores <- fread(here("data", "test", "noorderpoort_scores_by_topic.csv"))
np_test_scores[, Email := tolower(trimws(Email))]
# Link to MemoryLab user IDs
np_test_scores <- merge(np_test_scores, np_users, by.x = "Email", by.y = "email", all = TRUE)
There are some test scores for which we don’t have any associated
MemoryLab data:
np_test_scores[is.na(user_id), .(unique(Email))]
There are also some MemoryLab users for which we don’t have any
associated test scores:
np_test_scores[is.na(component), .(unique(Email))]
For this analysis we’ll only include users of whom we have two test
scores as well as some MemoryLab practice data.
np_test_scores[, did_ml := !is.na(user_id)]
np_test_scores[, two_tests := uniqueN(test) == 2, by = .(user_id)]
np_test_scores[, include_user := did_ml & two_tests]
Mean test scores from included students:
Distribution of test scores:

Do a paired t-test to show that the difference is significant:
t.test(test_score_dat$Posttest, test_score_dat$Pretest, paired = TRUE)
Paired t-test
data: test_score_dat$Posttest and test_score_dat$Pretest
t = -5.2446, df = 73, p-value = 1.47e-06
alternative hypothesis: true mean difference is not equal to 0
95 percent confidence interval:
-2.741364 -1.231609
sample estimates:
mean difference
-1.986486
Combine data:
np_scores <- np_test_scores[include_user == TRUE & !component %in% c("Totaal punten", "Cijfer"), .(
user_id,
topic = component,
score,
test
)]
np_scores <- dcast(np_scores, user_id + topic ~ test, value.var = "score")
setnames(np_scores, c("Posttest", "Pretest"), c("score_test_2", "score_test_1"))
np_scores[, score_test_change := score_test_2 - score_test_1]
# np_scores[, topic := factor(topic, levels = c("Delen",
# "Percentage",
# "Cijfers",
# "Breuken",
# "Tafels",
# "Decimalen",
# "Aftrekken & Optellen",
# "Vermenigvuldigen",
# "Afronden",
# "Eenheden",
# "Rekentaal"))]
np_scores_and_practice <- merge(np_scores, np_practice_stats, by = c("user_id", "topic"), all.x = TRUE)
# If a user has no practice data, we'll fill in zeros
np_scores_and_practice[is.na(n_sessions), n_sessions := 0]
np_scores_and_practice[is.na(n_responses), n_responses := 0]
np_scores_and_practice[is.na(duration), duration := 0]
Plot of scores:
mean_scores <- np_scores_and_practice[, .(score_test_1 = mean(score_test_1), score_test_2 = mean(score_test_2)), by = .(topic)] |>
melt(id.vars = "topic", variable.name = "test", value.name = "score")
p_scores <- melt(np_scores_and_practice, measure.vars = c("score_test_1", "score_test_2"), variable.name = "test", value.name = "score") |>
ggplot(aes(x = test, y = score)) +
facet_wrap(~ topic, ncol = 5) +
geom_point(alpha = .4, size = .5) +
geom_line(aes(group = user_id), alpha = .4, lty = 3) +
geom_point(data = mean_scores, colour = colours_memorylab[1], size = 2.5) +
geom_line(data = mean_scores, aes(group = topic), colour = colours_memorylab[1], lwd = 1) +
scale_x_discrete(labels = c("1", "2")) +
labs(x = "Toetsmoment", y = "Score", title = "Toetsscores") +
theme_ml() +
theme(panel.grid.major.y = element_line(colour = "grey90"),
strip.text = element_text(face = "bold")
)
p_scores
ggsave(here("output", "testscores_noorderpoort.png"), width = 8, height = 5)

How much was each topic practiced?
ggplot(np_scores_and_practice, aes(x = n_responses)) +
facet_wrap(~ topic, ncol = 5) +
geom_histogram(binwidth = 20, fill = colours_memorylab[1]) +
labs(x = "Number of practice responses per student", y = "Frequency", colour = "Topic", caption = "Data from noorderpoort.memorylab.app") +
theme_ml()

Did students choose to practice topics on which their pretest score
was low?
# Add mean pretest scores per topic
mean_pretest_scores <- mean_scores[test == "score_test_1", .(topic, mean_score = round(score, 2))]
np_scores_and_practice <- merge(np_scores_and_practice, mean_pretest_scores, by = "topic")
np_scores_and_practice[, topic_label := paste0(topic, "\n(Gemiddelde score: ", mean_score, ")")]
ggplot(np_scores_and_practice, aes(x = score_test_1, y = n_responses)) +
facet_wrap(~ topic_label, ncol = 5) +
geom_point(aes(fill = as.factor(score_test_1)), colour = "black", alpha = .8, position = position_jitter(height = 0, width = .1, seed = 0), pch = 21) +
scale_fill_brewer(palette = "RdYlGn") +
guides(fill = "none") +
labs(x = "Score op Toets 1", y = "Aantal gemaakte MemoryLab oefeningen", colour = "Onderwerp", caption = "noorderpoort.memorylab.app") +
theme_ml() +
theme(panel.grid.major.x = element_line(colour = "grey90"),
panel.grid.major.y = element_line(colour = "grey90"),
strip.text = element_text(face = "bold"))

Interpretation: not really.
Same plot but with totals instead of individual values:
p_practice <- np_scores_and_practice[, .(n_sessions_total = sum(n_sessions)), by = .(topic_label, score_test_1)] |>
ggplot(aes(x = score_test_1, y = n_sessions_total)) +
facet_wrap(~ topic_label, ncol = 5) +
geom_col(aes(fill = as.factor(score_test_1)), colour = "black", alpha = .8) +
scale_fill_brewer(palette = "RdYlGn") +
guides(fill = "none") +
labs(x = "Score op Toets 1", y = "Aantal MemoryLab oefensessies", colour = "Onderwerp", title = "Oefenactiviteit") +
theme_ml() +
theme(panel.grid.major.x = element_line(colour = "grey90"),
panel.grid.major.y = element_line(colour = "grey90"),
strip.text = element_text(face = "bold"))
p_practice
ggsave(here("output", "memorylab_oefensessies_noorderpoort.png"), width = 9, height = 5)

Combined plot:
library(patchwork)
p_scores + p_practice + plot_layout(ncol = 1)
ggsave(here("output", "memorylab_oefening_en_scores_noorderpoort.png"), width = 10, height = 10)

Is there a relation between score change and the number of practice
sessions?
np_scores_and_practice[, .(n_sessions = sum(n_sessions), score_test_change = mean(score_test_change)), by = .(topic_label)]
ggplot(np_scores_and_practice, aes(x = n_sessions, y = score_test_change)) +
facet_wrap(~ topic) +
geom_smooth(method = "lm", colour = colours_memorylab[1]) +
geom_point(alpha = .25) +
labs(x = "Number of practice sessions", y = "Change in test score", colour = "Topic", caption = "Data from noorderpoort.memorylab.app") +
scale_colour_viridis_d() +
theme_ml()

It looks like there might be a positive effect of practice. Let’s
look at it more simply: Is there a relation between score change and
whether or not the student has practiced the topic at all?

Average change:
np_scores_and_practice[, .(N = .N, mean_score_test_change = mean(score_test_change), sd_score_test_change = sd(score_test_change)), by = .(topic, topic_label, did_practice)]
Is there a significant difference in score change between students
who practiced and those who didn’t, taking into account differences in
score on the first test?
library(lmerTest)
lmer(score_test_change ~ did_practice*score_test_1 + (1 | user_id), data = np_scores_and_practice) |>
summary()
Linear mixed model fit by REML. t-tests use Satterthwaite's method ['lmerModLmerTest']
Formula: score_test_change ~ did_practice * score_test_1 + (1 | user_id)
Data: np_scores_and_practice
REML criterion at convergence: 1900.3
Scaled residuals:
Min 1Q Median 3Q Max
-3.9140 -0.5643 0.2549 0.5545 3.0879
Random effects:
Groups Name Variance Std.Dev.
user_id (Intercept) 0.03115 0.1765
Residual 0.72498 0.8515
Number of obs: 740, groups: user_id, 74
Fixed effects:
Estimate Std. Error df t value Pr(>|t|)
(Intercept) 0.73169 0.17700 728.08196 4.134 3.98e-05 ***
did_practiceTRUE 0.95870 0.32490 732.05595 2.951 0.00327 **
score_test_1 -0.30359 0.05005 735.64156 -6.066 2.09e-09 ***
did_practiceTRUE:score_test_1 -0.17258 0.09070 733.75382 -1.903 0.05746 .
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
Correlation of Fixed Effects:
(Intr) dd_TRUE scr__1
dd_prctTRUE -0.537
score_tst_1 -0.968 0.526
dd_TRUE:__1 0.533 -0.978 -0.551
Yes: practicing is associated with an increase in score change of
.96; scoring a point higher on test 1 is associated with a lower score
change (-.30).
lmer(score_test_change ~ did_practice*topic + (1 | user_id), data = np_scores_and_practice) |>
summary()
Linear mixed model fit by REML. t-tests use Satterthwaite's method ['lmerModLmerTest']
Formula: score_test_change ~ did_practice * topic + (1 | user_id)
Data: np_scores_and_practice
REML criterion at convergence: 1879.5
Scaled residuals:
Min 1Q Median 3Q Max
-4.4845 -0.4243 0.0835 0.3943 3.7491
Random effects:
Groups Name Variance Std.Dev.
user_id (Intercept) 0.03031 0.1741
Residual 0.69869 0.8359
Number of obs: 740, groups: user_id, 74
Fixed effects:
Estimate Std. Error df t value Pr(>|t|)
(Intercept) -0.35524 0.15045 719.99346 -2.361 0.01848 *
did_practiceTRUE 0.64970 0.19920 714.24055 3.261 0.00116 **
topicAftrekken & Optellen 0.23774 0.20291 678.50225 1.172 0.24177
topicBreuken 0.28800 0.19954 683.33089 1.443 0.14939
topicCijfers -0.05404 0.18621 677.28939 -0.290 0.77175
topicDecimalen 0.29335 0.18096 687.94394 1.621 0.10546
topicDelen 0.26533 0.18722 687.45604 1.417 0.15686
topicEenheden -0.81664 0.17952 683.22964 -4.549 6.38e-06 ***
topicPercentage 0.04765 0.19666 689.08359 0.242 0.80861
topicTafels 0.42404 0.23702 696.66575 1.789 0.07404 .
topicVermenigvuldigen 0.31759 0.18590 684.84812 1.708 0.08802 .
did_practiceTRUE:topicAftrekken & Optellen -0.60389 0.27897 698.77735 -2.165 0.03075 *
did_practiceTRUE:topicBreuken -0.23866 0.28000 706.18970 -0.852 0.39432
did_practiceTRUE:topicCijfers -0.52268 0.30284 700.47656 -1.726 0.08480 .
did_practiceTRUE:topicDecimalen -0.85266 0.39312 719.27378 -2.169 0.03041 *
did_practiceTRUE:topicDelen -0.51008 0.30142 715.64178 -1.692 0.09103 .
did_practiceTRUE:topicEenheden -0.46998 0.47989 714.68596 -0.979 0.32774
did_practiceTRUE:topicPercentage -0.52837 0.28220 713.57571 -1.872 0.06157 .
did_practiceTRUE:topicTafels -0.76463 0.29569 711.92758 -2.586 0.00991 **
did_practiceTRUE:topicVermenigvuldigen -0.83879 0.30763 713.07137 -2.727 0.00656 **
---
Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1
Correlation matrix not shown by default, as p = 20 > 12.
Use print(x, correlation=TRUE) or
vcov(x) if you need it
On some topics, performance on the pretest was already really high,
in which case we would not expect much improvement from practice. Let’s
look at the relation between pretest score and score change, taking into
account whether the student practiced or not:
ggplot(np_scores_and_practice, aes(x = score_test_1, y = score_test_2, colour = did_practice)) +
facet_wrap(~ topic, ncol = 5) +
geom_abline(intercept = 0, slope = 1, linetype = 2) +
geom_smooth(method = "lm") +
geom_point(alpha = .25) +
labs(x = "Pretest score", y = "Posttest score", colour = "Did the student\npractice the topic?", caption = "Data from noorderpoort.memorylab.app") +
theme_ml() +
coord_fixed(xlim = c(0, 4), ylim = c(0, 4))

Let’s look at performance during practice. Accuracy by topic:
ggplot(np_practice_stats, aes(x = as.character(topic), y = accuracy, fill = topic)) +
geom_boxplot() +
geom_jitter(width = .1, height = 0, alpha = .5) +
labs(x = "Topic", y = "Accuracy", fill = "Topic") +
scale_y_continuous(limits = c(.4, 1), labels = scales::percent) +
scale_fill_viridis_d() +
guides(fill = "none") +
theme_ml() +
theme(panel.grid.major.y = element_line(colour = "grey"))

Accuracy by user:
np_practice_stats[n_responses > 10, .(mean_accuracy = mean(accuracy), sd_accuracy = sd(accuracy)), by = .(user_id)]
ggplot(np_practice_stats[n_responses > 10], aes(x = reorder(as.character(user_id), accuracy), y = accuracy)) +
geom_boxplot(outlier.shape = NA) +
geom_jitter(aes(colour = as.character(topic)), width = .1, height = 0, alpha = .25) +
labs(x = "Student", y = "Accuratesse", colour = "Onderwerp") +
scale_y_continuous(limits = c(.4, 1), labels = scales::percent) +
scale_colour_viridis_d() +
guides(fill = "none") +
theme_ml() +
theme(panel.grid.major.y = element_line(colour = "grey"),
axis.text.x = element_blank(),
axis.ticks.x = element_blank())

Speed of forgetting by fact and topic:
np_sof <- np_responses[, .(final_alpha = alpha[which.max(presentation_start_time)]), by = .(text, topic, user_id)]
Error in `[.data.table`(np_responses, , .(final_alpha = alpha[which.max(presentation_start_time)]), :
column or expression 1 of 'by' or 'keyby' is type closure. Do not quote column names. Usage: DT[,sum(colC),by=list(colA,month(colB))]
ggplot(np_sof_avg[N > 10], aes(x = sof_mean, y = tidytext::reorder_within(text, sof_mean, as.character(topic)), alpha = N)) +
facet_grid(as.character(topic) ~ ., scales = "free_y") +
geom_errorbarh(aes(xmin = sof_mean - sof_se, xmax = sof_mean + sof_se), height = 0, colour = colours_memorylab[5]) +
geom_point(colour = colours_memorylab[5]) +
labs(y = "Feit", x = "Vergeetsnelheid (hoger = moeilijker)", alpha = "Geoefend door\naantal studenten") +
theme_ml() +
scale_x_continuous(limits = c(.1, .5)) +
tidytext::scale_y_reordered() +
theme(axis.text.y = element_text(size = 4),
panel.grid.major.x = element_line(colour = "grey90"))

ggsave(here("output", "sof_by_fact_and_topic.png"), height = 15, width = 8)
Alfa college
alfa_domain <- query_db("SELECT * FROM domain WHERE name = 'alfa.memorylab.app';", database = "slimstampen")
Users registered on this domain:
alfa_users <- query_db(paste0("SELECT id AS user_id FROM users WHERE domain_id = ", alfa_domain$id, ";"), database = "slimstampen")
Lessons on this domain:
alfa_lessons <- query_db(paste0("SELECT * FROM lesson WHERE domain_id = ", alfa_domain$id, ";"), database = "slimstampen")
Sessions from users on this domain during the pilot period:
alfa_sessions <- query_db(paste0("SELECT * FROM session WHERE token_id = 2 AND user_id IN (", paste(alfa_users$user_id, collapse = ", "), ");"), database = "ssaas")
alfa_sessions <- alfa_sessions[create_time > "2024-11-01"]
When were these sessions?
alfa_sessions[, session_date := as.Date(create_time)]
ggplot(alfa_sessions, aes(x = session_date)) +
geom_histogram(binwidth = 1, fill = colours_memorylab[1]) +
labs(x = "Date", y = "Sessions per day", caption = "Data from alfa.memorylab.app") +
scale_y_continuous(expand = c(0, 0)) +
theme_ml() +
theme(panel.grid.major.y = element_line(colour = "grey90"))
Most popular days:
alfa_sessions[, .N, by = .(session_date)][order(-N)]
Total sessions:
nrow(alfa_sessions)
Sessions per user:
alfa_sessions[, .N, by = user_id][order(-N)]
Total users with at least one session:
length(unique(alfa_sessions$user_id))
Which lessons did users do? Parse the session context:
alfa_sessions_parsed <- alfa_sessions[, map_dfr(context, fromJSON)] |> setDT()
alfa_sessions_parsed[, .(`Keer geoefend` = .N), by = .(Les = title)][order(-`Keer geoefend`)] |> knitr::kable()
Mastery credits:
alfa_credits <- query_db(paste0("SELECT * FROM lesson_mastered WHERE user_id IN (", paste(alfa_users$user_id, collapse = ", "), ");"), database = "slimstampen")
Add lesson titles:
alfa_credits <- merge(alfa_credits, alfa_lessons[, .(lesson_id = id, title)])
alfa_credits[, .N, by = .(title)][order(-N)]
query_db(query = "SELECT *
FROM pg_catalog.pg_tables
WHERE schemaname != 'pg_catalog' AND
schemaname != 'information_schema';",
database = "slimstampen")
LS0tCnRpdGxlOiAiVXNhZ2Ugc3RhdGlzdGljcyIKc3VidGl0bGU6ICJNQk8gcmVrZW5lbiBwaWxvdCAyMDI0IgphdXRob3I6ICJNYWFydGVuIHZhbiBkZXIgVmVsZGUiCmRhdGU6ICJMYXN0IHVwZGF0ZWQ6IGByIFN5cy5EYXRlKClgIgpvdXRwdXQ6CiAgaHRtbF9ub3RlYm9vazoKICAgIHNtYXJ0OiBubwogICAgdG9jOiB5ZXMKICAgIHRvY19mbG9hdDogeWVzCiAgZ2l0aHViX2RvY3VtZW50OgogICAgdG9jOiB5ZXMKZWRpdG9yX29wdGlvbnM6IAogIGNodW5rX291dHB1dF90eXBlOiBpbmxpbmUKLS0tCgpgYGB7cn0KbGlicmFyeShoZXJlKQpsaWJyYXJ5KGRhdGEudGFibGUpCmxpYnJhcnkoZ2dwbG90MikKbGlicmFyeShqc29ubGl0ZSkKbGlicmFyeShwdXJycikKCnRoZW1lX21lbW9yeWxhYl91cmwgPC0gImh0dHBzOi8vcmF3LmdpdGh1YnVzZXJjb250ZW50LmNvbS9TbGltU3RhbXBlbi90aGVtZV9tZW1vcnlsYWIvbWFzdGVyL3RoZW1lX21lbW9yeWxhYi5SIgpzb3VyY2UodGhlbWVfbWVtb3J5bGFiX3VybCkKCnNvdXJjZShoZXJlKCIuLiIsICJkYXRhYmFzZXMiLCAiZGF0YWJhc2VfZnVuY3Rpb25zLlIiKSkKYGBgCgojIE5vb3JkZXJwb29ydAoKYGBge3J9Cm5wX2RvbWFpbiA8LSBxdWVyeV9kYigiU0VMRUNUICogRlJPTSBkb21haW4gV0hFUkUgbmFtZSA9ICdub29yZGVycG9vcnQubWVtb3J5bGFiLmFwcCc7IiwgZGF0YWJhc2UgPSAic2xpbXN0YW1wZW4iKQpgYGAKClVzZXJzIHJlZ2lzdGVyZWQgb24gdGhpcyBkb21haW46CmBgYHtyfQpucF91c2VycyA8LSBxdWVyeV9kYihwYXN0ZTAoIlNFTEVDVCBpZCBBUyB1c2VyX2lkLCBlbWFpbCBGUk9NIHVzZXJzIFdIRVJFIGRvbWFpbl9pZCA9ICIsIG5wX2RvbWFpbiRpZCwgIjsiKSwgZGF0YWJhc2UgPSAic2xpbXN0YW1wZW4iKQpgYGAKCkxlc3NvbnMgb24gdGhpcyBkb21haW46CmBgYHtyfQpucF9sZXNzb25zIDwtIHF1ZXJ5X2RiKHBhc3RlMCgiU0VMRUNUICogRlJPTSBsZXNzb24gV0hFUkUgZG9tYWluX2lkID0gIiwgbnBfZG9tYWluJGlkLCAiOyIpLCBkYXRhYmFzZSA9ICJzbGltc3RhbXBlbiIpCmBgYAoKU2Vzc2lvbnMgZnJvbSB1c2VycyBvbiB0aGlzIGRvbWFpbiBkdXJpbmcgdGhlIHBpbG90IHBlcmlvZDoKYGBge3J9Cm5wX3Nlc3Npb25zIDwtIHF1ZXJ5X2RiKHBhc3RlMCgiU0VMRUNUICogRlJPTSBzZXNzaW9uIFdIRVJFIHRva2VuX2lkID0gMiBBTkQgdXNlcl9pZCBJTiAoIiwgcGFzdGUobnBfdXNlcnMkdXNlcl9pZCwgY29sbGFwc2UgPSAiLCAiKSwgIik7IiksIGRhdGFiYXNlID0gInNzYWFzIikKbnBfc2Vzc2lvbnMgPC0gbnBfc2Vzc2lvbnNbY3JlYXRlX3RpbWUgPiAiMjAyNC0xMC0wNiJdW2NyZWF0ZV90aW1lIDwgIjIwMjQtMTEtMDUiXQpgYGAKCldoZW4gd2VyZSB0aGVzZSBzZXNzaW9ucz8KYGBge3J9Cm5wX3Nlc3Npb25zWywgc2Vzc2lvbl9kYXRlIDo9IGFzLkRhdGUoY3JlYXRlX3RpbWUpXQoKbnBfdGVzdF9kYXRlcyA8LSBkYXRhLnRhYmxlKHRlc3QgPSBjKCJUb2V0cyAxIiwgIlRvZXRzIDIiKSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgIGRhdGUgPSBhcy5EYXRlKGMoIjIwMjQtMTAtMDciLCAiMjAyNC0xMS0wNCIpKSkKCmdncGxvdChucF9zZXNzaW9ucywgYWVzKHggPSBzZXNzaW9uX2RhdGUpKSArCiAgZ2VvbV92bGluZShkYXRhID0gbnBfdGVzdF9kYXRlcywgYWVzKHhpbnRlcmNlcHQgPSBkYXRlKSwgbGluZXR5cGUgPSAiZGFzaGVkIiwgY29sb3VyID0gImdyZXk1MCIpICsKICBnZW9tX2xhYmVsKGRhdGEgPSBucF90ZXN0X2RhdGVzLCBhZXMoeCA9IGRhdGUsIHkgPSBJbmYsIGxhYmVsID0gdGVzdCksIHZqdXN0ID0gMS4wNSwgaGp1c3QgPSAuNSwgY29sb3VyID0gImdyZXk1MCIpICsKICBnZW9tX2hpc3RvZ3JhbShiaW53aWR0aCA9IDEsIGZpbGwgPSBjb2xvdXJzX21lbW9yeWxhYlsxXSkgKwogIGxhYnMoeCA9ICJEYXR1bSIsIHkgPSAiTWVtb3J5TGFiIG9lZmVuc2Vzc2llcyBwZXIgZGFnIiwgdGl0bGUgPSAiTWVtb3J5TGFiIG9lZmVuYWN0aXZpdGVpdCBvdmVyIGRlIHRpamQiKSArCiAgc2NhbGVfeF9kYXRlKGRhdGVfbGFiZWxzID0gIiVkLyVtIiwgZGF0ZV9icmVha3MgPSAiMSB3ZWVrIiwgbGltaXRzID0gYyhhcy5EYXRlKCIyMDI0LTEwLTA2IiksIGFzLkRhdGUoIjIwMjQtMTEtMDUiKSkpICsKICBzY2FsZV95X2NvbnRpbnVvdXMoZXhwYW5kID0gYygwLCAwKSkgKwogIHRoZW1lX21sKCkgKwogIHRoZW1lKHBhbmVsLmdyaWQubWFqb3IueSA9IGVsZW1lbnRfbGluZShjb2xvdXIgPSAiZ3JleTkwIikpCgpnZ3NhdmUoaGVyZSgib3V0cHV0IiwgIm1lbW9yeWxhYl9vZWZlbnNlc3NpZXNfb3Zlcl90aWpkLnBuZyIpLCB3aWR0aCA9IDEwLCBoZWlnaHQgPSA1KQpgYGAKCmBgYHtyfQpgYGAKCk1vc3QgcG9wdWxhciBkYXlzOgpgYGB7cn0KbnBfc2Vzc2lvbnNbLCAuTiwgYnkgPSAuKHNlc3Npb25fZGF0ZSldW29yZGVyKC1OKV0KYGBgClRvdGFsIHNlc3Npb25zOgpgYGB7cn0KbnJvdyhucF9zZXNzaW9ucykKYGBgCgpTZXNzaW9ucyBwZXIgdXNlcjoKYGBge3J9Cm5wX3Nlc3Npb25zWywgLk4sIGJ5ID0gdXNlcl9pZF1bb3JkZXIoLU4pXQpgYGAKClRvdGFsIHVzZXJzIHdpdGggYXQgbGVhc3Qgb25lIHNlc3Npb246CmBgYHtyfQpsZW5ndGgodW5pcXVlKG5wX3Nlc3Npb25zJHVzZXJfaWQpKQpgYGAKCgpXaGljaCBsZXNzb25zIGRpZCB1c2VycyBkbz8gUGFyc2UgdGhlIHNlc3Npb24gY29udGV4dDoKYGBge3J9Cm5wX3Nlc3Npb25zWywgY29udGV4dF9wYXJzZWQgOj0gbWFwKGNvbnRleHQsIGZ1bmN0aW9uICh4KSB7CiAgeCA8LSBmcm9tSlNPTih4KQogIGRhdGEudGFibGUobGVzc29uX2lkID0geCRsZXNzb25JZCwKICAgICAgICAgICAgIGxlc3Nvbl9ncm91cF9pZCA9IHgkbGVzc29uR3JvdXBJZCwKICAgICAgICAgICAgIHRpdGxlID0geCR0aXRsZSkKfSldCgpucF9zZXNzaW9uc19wYXJzZWQgPC0gbnBfc2Vzc2lvbnNbLCByYmluZGxpc3QoY29udGV4dF9wYXJzZWQpLCBieSA9IC4oc2Vzc2lvbl9pZCA9IGlkLCB1c2VyX2lkLCBjcmVhdGVfdGltZSwgc2Vzc2lvbl9kYXRlKV0KYGBgCgpQcmFjdGljZSBieSBsZXNzb246CmBgYHtyfQpucF9zZXNzaW9uc19wYXJzZWRbLCAuKGBLZWVyIGdlb2VmZW5kYCA9IC5OKSwgYnkgPSAuKExlcyA9IHRpdGxlKV1bb3JkZXIoLWBLZWVyIGdlb2VmZW5kYCldIHw+IGtuaXRyOjprYWJsZSgpCmBgYAoKV2UgY2FuIGxpbmsgZWFjaCBwcmFjdGljZSBzZXNzaW9uIHRvIG9uZSBvZiB0aGUgdGVzdCB0b3BpY3M6CmBgYHtyfQpucF9sZXNzb25fZ3JvdXBzIDwtIGRhdGEudGFibGUoCiAgbGVzc29uX2dyb3VwX2lkID0gYygKICAgIDI5MDkyLAogICAgMjkwODcsCiAgICAyOTA5NiwKICAgIDI5MDg5LAogICAgMjkwOTgsCiAgICAyOTA4NiwKICAgIDI5MDk3LAogICAgMjkwOTUsCiAgICAyOTA5MCwKICAgIDI5MDg4LAogICAgMjkwOTQsCiAgICAyOTA5MSksCiAgdG9waWMgPSBjKAogICAgIlJla2VudGFhbCIsCiAgICAiQnJldWtlbiIsCiAgICAiUGVyY2VudGFnZSIsCiAgICAiRWVuaGVkZW4iLAogICAgIkRlbGVuIiwKICAgICJBZnJvbmRlbiIsCiAgICAiQ2lqZmVycyIsCiAgICAiVmVybWVuaWd2dWxkaWdlbiIsCiAgICAiQWZ0cmVra2VuICYgT3B0ZWxsZW4iLAogICAgIkRlY2ltYWxlbiIsCiAgICAiVGFmZWxzIiwKICAgICJSZWtlbnRhYWwiCiAgKQopCgojIFNldCBsZXNzb24gdG9waWNzIHRvIHRoZSByaWdodCBvcmRlcgpucF9sZXNzb25fZ3JvdXBzWywgdG9waWMgOj0gZmFjdG9yKHRvcGljLCBsZXZlbHMgPSBjKCJEZWxlbiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIlBlcmNlbnRhZ2UiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJDaWpmZXJzIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiQnJldWtlbiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIlRhZmVscyIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIkRlY2ltYWxlbiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIkFmdHJla2tlbiAmIE9wdGVsbGVuIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiVmVybWVuaWd2dWxkaWdlbiIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIkFmcm9uZGVuIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiRWVuaGVkZW4iLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJSZWtlbnRhYWwiKSldCgoKbnBfbGVzc29ucyA8LSBtZXJnZShucF9sZXNzb25zLCBucF9sZXNzb25fZ3JvdXBzLCBieSA9ICJsZXNzb25fZ3JvdXBfaWQiKQpucF9zZXNzaW9uc19wYXJzZWQgPC0gbWVyZ2UobnBfc2Vzc2lvbnNfcGFyc2VkLCBucF9sZXNzb25fZ3JvdXBzLCBieSA9ICJsZXNzb25fZ3JvdXBfaWQiKQpgYGAKClByYWN0aWNlIGJ5IGxlc3NvbiB0b3BpYzoKYGBge3J9Cm5wX3Nlc3Npb25zX3BhcnNlZFssIC4oYEtlZXIgZ2VvZWZlbmRgID0gLk4pLCBieSA9IC4oT25kZXJ3ZXJwID0gdG9waWMpXVtvcmRlcigtYEtlZXIgZ2VvZWZlbmRgKV0gfD4ga25pdHI6OmthYmxlKCkKYGBgCgpCYXIgcGxvdCBvZiBzZXNzaW9ucyBwZXIgdG9waWM6CmBgYHtyfQpucF9zZXNzaW9uc19wYXJzZWRbLCAuKGBLZWVyIGdlb2VmZW5kYCA9IC5OKSwgYnkgPSAuKE9uZGVyd2VycCA9IHRvcGljKV0gfD4KICBnZ3Bsb3QoYWVzKHggPSBPbmRlcndlcnAsIHkgPSBgS2VlciBnZW9lZmVuZGApKSArCiAgZ2VvbV9jb2woZmlsbCA9IGNvbG91cnNfbWVtb3J5bGFiWzFdKSArCiAgbGFicyh4ID0gIk9uZGVyd2VycCIsIHkgPSAiS2VlciBnZW9lZmVuZCIsIGNhcHRpb24gPSAiRGF0YSBmcm9tIG5vb3JkZXJwb29ydC5tZW1vcnlsYWIuYXBwIikgKwogIHRoZW1lX21sKCkKYGBgCgoKTWFzdGVyeSBjcmVkaXRzOgpgYGB7cn0KbnBfY3JlZGl0cyA8LSBxdWVyeV9kYihwYXN0ZTAoIlNFTEVDVCAqIEZST00gbGVzc29uX21hc3RlcmVkIFdIRVJFIHVzZXJfaWQgSU4gKCIsIHBhc3RlKG5wX3VzZXJzJHVzZXJfaWQsIGNvbGxhcHNlID0gIiwgIiksICIpOyIpLCBkYXRhYmFzZSA9ICJzbGltc3RhbXBlbiIpCgojIEFkZCBsZXNzb24gdGl0bGVzIGFuZCB0b3BpY3MKbnBfY3JlZGl0cyA8LSBtZXJnZShucF9jcmVkaXRzLCBucF9sZXNzb25zWywgLihsZXNzb25faWQgPSBpZCwgdGl0bGUsIHRvcGljKV0pCmBgYAoKQ3JlZGl0cyBieSB0b3BpYzoKYGBge3J9Cm5wX2NyZWRpdHNbLCAuTiwgYnkgPSAuKHRvcGljKV1bb3JkZXIoLU4pXQpgYGAKClJlc3BvbnNlcyBmcm9tIHRoZXNlIHVzZXJfaWRzOgpgYGB7cn0KbnBfcmVzcG9uc2VzIDwtIHF1ZXJ5X2RiKHBhc3RlMCgiU0VMRUNUICogRlJPTSByZXNwb25zZSBXSEVSRSB0b2tlbl9pZCA9IDIgQU5EIHVzZXJfaWQgSU4gKCIsIHBhc3RlKG5wX3VzZXJzJHVzZXJfaWQsIGNvbGxhcHNlID0gIiwgIiksICIpOyIpLCBkYXRhYmFzZSA9ICJzc2FhcyIpCm5wX3Jlc3BvbnNlcyA8LSBucF9yZXNwb25zZXNbY3JlYXRlX3RpbWUgPiAiMjAyNC0xMC0wNiJdW2NyZWF0ZV90aW1lIDwgIjIwMjQtMTEtMDUiXQoKIyBBZGQgbGVzc29uIHRpdGxlcyBhbmQgdG9waWNzCm5wX3Jlc3BvbnNlcyA8LSBtZXJnZShucF9yZXNwb25zZXMsIG5wX3Nlc3Npb25zX3BhcnNlZFssIC4oc2Vzc2lvbl9pZCwgdGl0bGUsIHRvcGljKV0sIGJ5ID0gInNlc3Npb25faWQiKQpgYGAKClJlc3BvbnNlcyBieSB1c2VyOgpgYGB7cn0KbnBfcmVzcG9uc2VzWywgLk4sIGJ5ID0gLih1c2VyX2lkKV1bb3JkZXIoLU4pXQpgYGAKClJlc3BvbnNlcyBieSB0b3BpYzoKYGBge3J9Cm5wX3Jlc3BvbnNlc1ssIC5OLCBieSA9IC4odG9waWMpXVtvcmRlcigtTildCmBgYAoKCgoKCmBgYHtyfQpnZ3Bsb3QobnBfcmVzcG9uc2VzLCBhZXMoeCA9IGFzLkRhdGUoY3JlYXRlX3RpbWUpKSkgKwogIGdlb21faGlzdG9ncmFtKGJpbndpZHRoID0gMSwgZmlsbCA9IGNvbG91cnNfbWVtb3J5bGFiWzFdKSArCiAgbGFicyh4ID0gIkRhdGUiLCB5ID0gIlJlc3BvbnNlcyBwZXIgZGF5IiwgY2FwdGlvbiA9ICJEYXRhIGZyb20gbm9vcmRlcnBvb3J0Lm1lbW9yeWxhYi5hcHAiKSArCiAgc2NhbGVfeV9jb250aW51b3VzKGV4cGFuZCA9IGMoMCwgMCksIGxhYmVscyA9IHNjYWxlczo6bnVtYmVyX2Zvcm1hdChiaWcubWFyayA9ICIsIikpICsKICB0aGVtZV9tbCgpICsKICB0aGVtZShwYW5lbC5ncmlkLm1ham9yLnkgPSBlbGVtZW50X2xpbmUoY29sb3VyID0gImdyZXk5MCIpKQpgYGAKIApTcGxpdCBieSBsZXNzb24gdG9waWM6CmBgYHtyfQpnZ3Bsb3QobnBfcmVzcG9uc2VzLCBhZXMoeCA9IGFzLkRhdGUoY3JlYXRlX3RpbWUpKSkgKwogIGdlb21faGlzdG9ncmFtKGFlcyhmaWxsID0gdG9waWMpLCBiaW53aWR0aCA9IDEpICsKICBsYWJzKHggPSAiRGF0ZSIsIHkgPSAiUmVzcG9uc2VzIHBlciBkYXkiLCBmaWxsID0gIk9uZGVyd2VycCIsIGNhcHRpb24gPSAiRGF0YSBmcm9tIG5vb3JkZXJwb29ydC5tZW1vcnlsYWIuYXBwIikgKwogIHNjYWxlX3lfY29udGludW91cyhleHBhbmQgPSBjKDAsIDApLCBsYWJlbHMgPSBzY2FsZXM6Om51bWJlcl9mb3JtYXQoYmlnLm1hcmsgPSAiLCIpKSArCiAgc2NhbGVfZmlsbF92aXJpZGlzX2QoKSArCiAgdGhlbWVfbWwoKSArCiAgdGhlbWUocGFuZWwuZ3JpZC5tYWpvci55ID0gZWxlbWVudF9saW5lKGNvbG91ciA9ICJncmV5OTAiKSkKCmBgYAogCgojIyBIb3cgZG9lcyBzdHVkeSBiZWhhdmlvdXIgb24gc3BlY2lmaWMgdG9waWNzIHJlbGF0ZSB0byB0ZXN0IHBlcmZvcm1hbmNlPwoKV2Ugd2FudCB0byBzZWUgd2hldGhlciBzdHVkeWluZyBhIHNwZWNpZmljIHRvcGljIGlzIHJlbGF0ZWQgdG8gYW4gaW5jcmVhc2UgaW4gdGVzdCBwZXJmb3JtYW5jZS4KU3R1ZHlpbmcgYmVoYXZpb3VyIGNhbiBiZSBzdW1tYXJpc2VkIGluIHNldmVyYWwgd2F5czogdGltZSBzcGVudCwgbnVtYmVyIG9mIHNlc3Npb25zLCBudW1iZXIgb2YgcXVlc3Rpb25zIGFuc3dlcmVkLCBudW1iZXIgb2YgY3JlZGl0cyBhY2hpZXZlZC4KCmBgYHtyfQpucF9zZXNzaW9uX3N0YXRzIDwtIG5wX3Jlc3BvbnNlc1ssIC4oCiAgbl9yZXNwb25zZXMgPSAuTiwKICBkdXJhdGlvbiA9IG1heChwcmVzZW50YXRpb25fc3RhcnRfdGltZSkgKyBwcmVzZW50YXRpb25fZHVyYXRpb25bd2hpY2gubWF4KHByZXNlbnRhdGlvbl9zdGFydF90aW1lKV0gLSBtaW4ocHJlc2VudGF0aW9uX3N0YXJ0X3RpbWUpLAogIGFjY3VyYWN5ID0gbWVhbihjb3JyZWN0KQopLCBieSA9IC4odXNlcl9pZCwgdG9waWMsIHNlc3Npb25faWQpXQoKbnBfcHJhY3RpY2Vfc3RhdHMgPC0gbnBfc2Vzc2lvbl9zdGF0c1ssIC4oCiAgbl9zZXNzaW9ucyA9IC5OLAogIG5fcmVzcG9uc2VzID0gc3VtKG5fcmVzcG9uc2VzKSwKICBkdXJhdGlvbiA9IHN1bShkdXJhdGlvbiksCiAgYWNjdXJhY3kgPSBtZWFuKGFjY3VyYWN5KQopLCBieSA9IC4odXNlcl9pZCwgdG9waWMpXQpgYGAKCkxvYWQgdGVzdCBzY29yZXMgcGVyIHRvcGljOgpgYGB7cn0KbnBfdGVzdF9zY29yZXMgPC0gZnJlYWQoaGVyZSgiZGF0YSIsICJ0ZXN0IiwgIm5vb3JkZXJwb29ydF9zY29yZXNfYnlfdG9waWMuY3N2IikpCm5wX3Rlc3Rfc2NvcmVzWywgRW1haWwgOj0gdG9sb3dlcih0cmltd3MoRW1haWwpKV0KCiMgTGluayB0byBNZW1vcnlMYWIgdXNlciBJRHMKbnBfdGVzdF9zY29yZXMgPC0gbWVyZ2UobnBfdGVzdF9zY29yZXMsIG5wX3VzZXJzLCBieS54ID0gIkVtYWlsIiwgYnkueSA9ICJlbWFpbCIsIGFsbCA9IFRSVUUpCmBgYAoKVGhlcmUgYXJlIHNvbWUgdGVzdCBzY29yZXMgZm9yIHdoaWNoIHdlIGRvbid0IGhhdmUgYW55IGFzc29jaWF0ZWQgTWVtb3J5TGFiIGRhdGE6CmBgYHtyfQpucF90ZXN0X3Njb3Jlc1tpcy5uYSh1c2VyX2lkKSwgLih1bmlxdWUoRW1haWwpKV0KYGBgCgpUaGVyZSBhcmUgYWxzbyBzb21lIE1lbW9yeUxhYiB1c2VycyBmb3Igd2hpY2ggd2UgZG9uJ3QgaGF2ZSBhbnkgYXNzb2NpYXRlZCB0ZXN0IHNjb3JlczoKYGBge3J9Cm5wX3Rlc3Rfc2NvcmVzW2lzLm5hKGNvbXBvbmVudCksIC4odW5pcXVlKEVtYWlsKSldCmBgYAoKRm9yIHRoaXMgYW5hbHlzaXMgd2UnbGwgb25seSBpbmNsdWRlIHVzZXJzIG9mIHdob20gd2UgaGF2ZSB0d28gdGVzdCBzY29yZXMgYXMgd2VsbCBhcyBzb21lIE1lbW9yeUxhYiBwcmFjdGljZSBkYXRhLgpgYGB7cn0KbnBfdGVzdF9zY29yZXNbLCBkaWRfbWwgOj0gIWlzLm5hKHVzZXJfaWQpXQpucF90ZXN0X3Njb3Jlc1ssIHR3b190ZXN0cyA6PSB1bmlxdWVOKHRlc3QpID09IDIsIGJ5ID0gLih1c2VyX2lkKV0KbnBfdGVzdF9zY29yZXNbLCBpbmNsdWRlX3VzZXIgOj0gZGlkX21sICYgdHdvX3Rlc3RzXQpgYGAKCk1lYW4gdGVzdCBzY29yZXMgZnJvbSBpbmNsdWRlZCBzdHVkZW50czoKYGBge3J9Cm5wX3Rlc3Rfc2NvcmVzW2luY2x1ZGVfdXNlciA9PSBUUlVFICYgY29tcG9uZW50ID09ICJUb3RhYWwgcHVudGVuIiwgLihtZWFuX2dyYWRlID0gMTAqbWVhbihzY29yZSkvNDApLCBieSA9IHRlc3RdCmBgYAoKRGlzdHJpYnV0aW9uIG9mIHRlc3Qgc2NvcmVzOgpgYGB7cn0KbnBfdGVzdF9zY29yZXNbaW5jbHVkZV91c2VyID09IFRSVUUgJiBjb21wb25lbnQgPT0gIlRvdGFhbCBwdW50ZW4iLCAuKHRlc3QsIHNjb3JlKV0gfD4KICBnZ3Bsb3QoYWVzKHggPSAxMCpzY29yZS80MCwgZmlsbCA9IHRlc3QpKSArCiAgZ2VvbV9kZW5zaXR5KGFscGhhID0gMC41KSArCiAgbGFicyh4ID0gIlNjb3JlIiwgeSA9ICJEZW5zaXR5IiwgZmlsbCA9ICJUZXN0IiwgY2FwdGlvbiA9ICJEYXRhIGZyb20gbm9vcmRlcnBvb3J0Lm1lbW9yeWxhYi5hcHAiKSArCiAgc2NhbGVfeF9jb250aW51b3VzKGJyZWFrcyA9IHNlcSgwLCAxMCwgMSksIGxpbWl0cyA9IGMoMCwgMTApKSArCiAgdGhlbWVfbWwoKQpgYGAKCkRvIGEgcGFpcmVkIHQtdGVzdCB0byBzaG93IHRoYXQgdGhlIGRpZmZlcmVuY2UgaXMgc2lnbmlmaWNhbnQ6CmBgYHtyfQp0ZXN0X3Njb3JlX2RhdCA8LSBucF90ZXN0X3Njb3Jlc1tpbmNsdWRlX3VzZXIgPT0gVFJVRSAmIGNvbXBvbmVudCA9PSAiVG90YWFsIHB1bnRlbiIsIC4odXNlcl9pZCwgdGVzdCwgc2NvcmUpXSB8PgogIGRjYXN0KHVzZXJfaWQgfiB0ZXN0LCB2YWx1ZS52YXIgPSAic2NvcmUiKQoKdGVzdF9zY29yZV9kYXQKCnQudGVzdCh0ZXN0X3Njb3JlX2RhdCRQb3N0dGVzdCwgdGVzdF9zY29yZV9kYXQkUHJldGVzdCwgcGFpcmVkID0gVFJVRSkKYGBgCgoKCkNvbWJpbmUgZGF0YToKYGBge3J9Cm5wX3Njb3JlcyA8LSBucF90ZXN0X3Njb3Jlc1tpbmNsdWRlX3VzZXIgPT0gVFJVRSAmICFjb21wb25lbnQgJWluJSBjKCJUb3RhYWwgcHVudGVuIiwgIkNpamZlciIpLCAuKAogIHVzZXJfaWQsCiAgdG9waWMgPSBjb21wb25lbnQsCiAgc2NvcmUsCiAgdGVzdAopXQpucF9zY29yZXMgPC0gZGNhc3QobnBfc2NvcmVzLCB1c2VyX2lkICsgdG9waWMgfiB0ZXN0LCB2YWx1ZS52YXIgPSAic2NvcmUiKQpzZXRuYW1lcyhucF9zY29yZXMsIGMoIlBvc3R0ZXN0IiwgIlByZXRlc3QiKSwgYygic2NvcmVfdGVzdF8yIiwgInNjb3JlX3Rlc3RfMSIpKQpucF9zY29yZXNbLCBzY29yZV90ZXN0X2NoYW5nZSA6PSBzY29yZV90ZXN0XzIgLSBzY29yZV90ZXN0XzFdCiMgbnBfc2NvcmVzWywgdG9waWMgOj0gZmFjdG9yKHRvcGljLCBsZXZlbHMgPSBjKCJEZWxlbiIsCiMgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJQZXJjZW50YWdlIiwKIyAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIkNpamZlcnMiLAojICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiQnJldWtlbiIsCiMgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJUYWZlbHMiLAojICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiRGVjaW1hbGVuIiwKIyAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIkFmdHJla2tlbiAmIE9wdGVsbGVuIiwKIyAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIlZlcm1lbmlndnVsZGlnZW4iLAojICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiQWZyb25kZW4iLAojICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiRWVuaGVkZW4iLAojICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiUmVrZW50YWFsIikpXQoKbnBfc2NvcmVzX2FuZF9wcmFjdGljZSA8LSAgbWVyZ2UobnBfc2NvcmVzLCBucF9wcmFjdGljZV9zdGF0cywgYnkgPSBjKCJ1c2VyX2lkIiwgInRvcGljIiksIGFsbC54ID0gVFJVRSkKCiMgSWYgYSB1c2VyIGhhcyBubyBwcmFjdGljZSBkYXRhLCB3ZSdsbCBmaWxsIGluIHplcm9zCm5wX3Njb3Jlc19hbmRfcHJhY3RpY2VbaXMubmEobl9zZXNzaW9ucyksIG5fc2Vzc2lvbnMgOj0gMF0KbnBfc2NvcmVzX2FuZF9wcmFjdGljZVtpcy5uYShuX3Jlc3BvbnNlcyksIG5fcmVzcG9uc2VzIDo9IDBdCm5wX3Njb3Jlc19hbmRfcHJhY3RpY2VbaXMubmEoZHVyYXRpb24pLCBkdXJhdGlvbiA6PSAwXQpgYGAKCgpQbG90IG9mIHNjb3JlczoKYGBge3J9Cm1lYW5fc2NvcmVzIDwtIG5wX3Njb3Jlc19hbmRfcHJhY3RpY2VbLCAuKHNjb3JlX3Rlc3RfMSA9IG1lYW4oc2NvcmVfdGVzdF8xKSwgc2NvcmVfdGVzdF8yID0gbWVhbihzY29yZV90ZXN0XzIpKSwgYnkgPSAuKHRvcGljKV0gfD4KICBtZWx0KGlkLnZhcnMgPSAidG9waWMiLCB2YXJpYWJsZS5uYW1lID0gInRlc3QiLCB2YWx1ZS5uYW1lID0gInNjb3JlIikKCnBfc2NvcmVzIDwtIG1lbHQobnBfc2NvcmVzX2FuZF9wcmFjdGljZSwgbWVhc3VyZS52YXJzID0gYygic2NvcmVfdGVzdF8xIiwgInNjb3JlX3Rlc3RfMiIpLCB2YXJpYWJsZS5uYW1lID0gInRlc3QiLCB2YWx1ZS5uYW1lID0gInNjb3JlIikgfD4KICBnZ3Bsb3QoYWVzKHggPSB0ZXN0LCB5ID0gc2NvcmUpKSArCiAgZmFjZXRfd3JhcCh+IHRvcGljLCBuY29sID0gNSkgKwogIGdlb21fcG9pbnQoYWxwaGEgPSAuNCwgc2l6ZSA9IC41KSArCiAgZ2VvbV9saW5lKGFlcyhncm91cCA9IHVzZXJfaWQpLCBhbHBoYSA9IC40LCBsdHkgPSAzKSArCiAgZ2VvbV9wb2ludChkYXRhID0gbWVhbl9zY29yZXMsIGNvbG91ciA9IGNvbG91cnNfbWVtb3J5bGFiWzFdLCBzaXplID0gMi41KSArCiAgZ2VvbV9saW5lKGRhdGEgPSBtZWFuX3Njb3JlcywgYWVzKGdyb3VwID0gdG9waWMpLCBjb2xvdXIgPSBjb2xvdXJzX21lbW9yeWxhYlsxXSwgbHdkID0gMSkgKwogIHNjYWxlX3hfZGlzY3JldGUobGFiZWxzID0gYygiMSIsICIyIikpICsKICBsYWJzKHggPSAiVG9ldHNtb21lbnQiLCB5ID0gIlNjb3JlIiwgdGl0bGUgPSAiVG9ldHNzY29yZXMiKSArCiAgdGhlbWVfbWwoKSArCiAgdGhlbWUocGFuZWwuZ3JpZC5tYWpvci55ID0gZWxlbWVudF9saW5lKGNvbG91ciA9ICJncmV5OTAiKSwKICAgICAgICBzdHJpcC50ZXh0ID0gZWxlbWVudF90ZXh0KGZhY2UgPSAiYm9sZCIpCiAgICAgICAgKQoKcF9zY29yZXMKCmdnc2F2ZShoZXJlKCJvdXRwdXQiLCAidGVzdHNjb3Jlc19ub29yZGVycG9vcnQucG5nIiksIHdpZHRoID0gOCwgaGVpZ2h0ID0gNSkKCmBgYAoKSG93IG11Y2ggd2FzIGVhY2ggdG9waWMgcHJhY3RpY2VkPwpgYGB7cn0KZ2dwbG90KG5wX3Njb3Jlc19hbmRfcHJhY3RpY2UsIGFlcyh4ID0gbl9yZXNwb25zZXMpKSArCiAgZmFjZXRfd3JhcCh+IHRvcGljLCBuY29sID0gNSkgKwogIGdlb21faGlzdG9ncmFtKGJpbndpZHRoID0gMjAsIGZpbGwgPSBjb2xvdXJzX21lbW9yeWxhYlsxXSkgKwogIGxhYnMoeCA9ICJOdW1iZXIgb2YgcHJhY3RpY2UgcmVzcG9uc2VzIHBlciBzdHVkZW50IiwgeSA9ICJGcmVxdWVuY3kiLCBjb2xvdXIgPSAiVG9waWMiLCBjYXB0aW9uID0gIkRhdGEgZnJvbSBub29yZGVycG9vcnQubWVtb3J5bGFiLmFwcCIpICsKICB0aGVtZV9tbCgpCmBgYAoKRGlkIHN0dWRlbnRzIGNob29zZSB0byBwcmFjdGljZSB0b3BpY3Mgb24gd2hpY2ggdGhlaXIgcHJldGVzdCBzY29yZSB3YXMgbG93PwpgYGB7cn0KIyBBZGQgbWVhbiBwcmV0ZXN0IHNjb3JlcyBwZXIgdG9waWMKbWVhbl9wcmV0ZXN0X3Njb3JlcyA8LSBtZWFuX3Njb3Jlc1t0ZXN0ID09ICJzY29yZV90ZXN0XzEiLCAuKHRvcGljLCBtZWFuX3Njb3JlID0gcm91bmQoc2NvcmUsIDIpKV0KbnBfc2NvcmVzX2FuZF9wcmFjdGljZSA8LSBtZXJnZShucF9zY29yZXNfYW5kX3ByYWN0aWNlLCBtZWFuX3ByZXRlc3Rfc2NvcmVzLCBieSA9ICJ0b3BpYyIpCm5wX3Njb3Jlc19hbmRfcHJhY3RpY2VbLCB0b3BpY19sYWJlbCA6PSBwYXN0ZTAodG9waWMsICJcbihHZW1pZGRlbGRlIHNjb3JlOiAiLCBtZWFuX3Njb3JlLCAiKSIpXQoKZ2dwbG90KG5wX3Njb3Jlc19hbmRfcHJhY3RpY2UsIGFlcyh4ID0gc2NvcmVfdGVzdF8xLCB5ID0gbl9yZXNwb25zZXMpKSArCiAgZmFjZXRfd3JhcCh+IHRvcGljX2xhYmVsLCBuY29sID0gNSkgKwogIGdlb21fcG9pbnQoYWVzKGZpbGwgPSBhcy5mYWN0b3Ioc2NvcmVfdGVzdF8xKSksIGNvbG91ciA9ICJibGFjayIsIGFscGhhID0gLjgsIHBvc2l0aW9uID0gcG9zaXRpb25faml0dGVyKGhlaWdodCA9IDAsIHdpZHRoID0gLjEsIHNlZWQgPSAwKSwgcGNoID0gMjEpICsKICBzY2FsZV9maWxsX2JyZXdlcihwYWxldHRlID0gIlJkWWxHbiIpICsKICBndWlkZXMoZmlsbCA9ICJub25lIikgKwogIGxhYnMoeCA9ICJTY29yZSBvcCBUb2V0cyAxIiwgeSA9ICJBYW50YWwgZ2VtYWFrdGUgTWVtb3J5TGFiIG9lZmVuaW5nZW4iLCBjb2xvdXIgPSAiT25kZXJ3ZXJwIiwgY2FwdGlvbiA9ICJub29yZGVycG9vcnQubWVtb3J5bGFiLmFwcCIpICsKICB0aGVtZV9tbCgpICsKICB0aGVtZShwYW5lbC5ncmlkLm1ham9yLnggPSBlbGVtZW50X2xpbmUoY29sb3VyID0gImdyZXk5MCIpLAogICAgICAgIHBhbmVsLmdyaWQubWFqb3IueSA9IGVsZW1lbnRfbGluZShjb2xvdXIgPSAiZ3JleTkwIiksCiAgICAgICAgc3RyaXAudGV4dCA9IGVsZW1lbnRfdGV4dChmYWNlID0gImJvbGQiKSkKYGBgCgpJbnRlcnByZXRhdGlvbjogbm90IHJlYWxseS4KCgpTYW1lIHBsb3QgYnV0IHdpdGggdG90YWxzIGluc3RlYWQgb2YgaW5kaXZpZHVhbCB2YWx1ZXM6CmBgYHtyfQpwX3ByYWN0aWNlIDwtIG5wX3Njb3Jlc19hbmRfcHJhY3RpY2VbLCAuKG5fc2Vzc2lvbnNfdG90YWwgPSBzdW0obl9zZXNzaW9ucykpLCBieSA9IC4odG9waWNfbGFiZWwsIHNjb3JlX3Rlc3RfMSldIHw+CiAgZ2dwbG90KGFlcyh4ID0gc2NvcmVfdGVzdF8xLCB5ID0gbl9zZXNzaW9uc190b3RhbCkpICsKICBmYWNldF93cmFwKH4gdG9waWNfbGFiZWwsIG5jb2wgPSA1KSArCiAgZ2VvbV9jb2woYWVzKGZpbGwgPSBhcy5mYWN0b3Ioc2NvcmVfdGVzdF8xKSksIGNvbG91ciA9ICJibGFjayIsIGFscGhhID0gLjgpICsKICBzY2FsZV9maWxsX2JyZXdlcihwYWxldHRlID0gIlJkWWxHbiIpICsKICBndWlkZXMoZmlsbCA9ICJub25lIikgKwogIGxhYnMoeCA9ICJTY29yZSBvcCBUb2V0cyAxIiwgeSA9ICJBYW50YWwgTWVtb3J5TGFiIG9lZmVuc2Vzc2llcyIsIGNvbG91ciA9ICJPbmRlcndlcnAiLCB0aXRsZSA9ICJPZWZlbmFjdGl2aXRlaXQiKSArCiAgdGhlbWVfbWwoKSArCiAgdGhlbWUocGFuZWwuZ3JpZC5tYWpvci54ID0gZWxlbWVudF9saW5lKGNvbG91ciA9ICJncmV5OTAiKSwKICAgICAgICBwYW5lbC5ncmlkLm1ham9yLnkgPSBlbGVtZW50X2xpbmUoY29sb3VyID0gImdyZXk5MCIpLAogICAgICAgIHN0cmlwLnRleHQgPSBlbGVtZW50X3RleHQoZmFjZSA9ICJib2xkIikpCgpwX3ByYWN0aWNlCgpnZ3NhdmUoaGVyZSgib3V0cHV0IiwgIm1lbW9yeWxhYl9vZWZlbnNlc3NpZXNfbm9vcmRlcnBvb3J0LnBuZyIpLCB3aWR0aCA9IDksIGhlaWdodCA9IDUpCmBgYAoKCkNvbWJpbmVkIHBsb3Q6CmBgYHtyfQpsaWJyYXJ5KHBhdGNod29yaykKCnBfc2NvcmVzICsgcF9wcmFjdGljZSArIHBsb3RfbGF5b3V0KG5jb2wgPSAxKQpnZ3NhdmUoaGVyZSgib3V0cHV0IiwgIm1lbW9yeWxhYl9vZWZlbmluZ19lbl9zY29yZXNfbm9vcmRlcnBvb3J0LnBuZyIpLCB3aWR0aCA9IDEwLCBoZWlnaHQgPSAxMCkKYGBgCgoKCklzIHRoZXJlIGEgcmVsYXRpb24gYmV0d2VlbiBzY29yZSBjaGFuZ2UgYW5kIHRoZSBudW1iZXIgb2YgcHJhY3RpY2Ugc2Vzc2lvbnM/CmBgYHtyfQpucF9zY29yZXNfYW5kX3ByYWN0aWNlWywgLihuX3Nlc3Npb25zID0gc3VtKG5fc2Vzc2lvbnMpLCBzY29yZV90ZXN0X2NoYW5nZSA9IG1lYW4oc2NvcmVfdGVzdF9jaGFuZ2UpKSwgYnkgPSAuKHRvcGljX2xhYmVsKV0KCmdncGxvdChucF9zY29yZXNfYW5kX3ByYWN0aWNlLCBhZXMoeCA9IG5fc2Vzc2lvbnMsIHkgPSBzY29yZV90ZXN0X2NoYW5nZSkpICsKICBmYWNldF93cmFwKH4gdG9waWMpICsKICBnZW9tX3Ntb290aChtZXRob2QgPSAibG0iLCBjb2xvdXIgPSBjb2xvdXJzX21lbW9yeWxhYlsxXSkgKwogIGdlb21fcG9pbnQoYWxwaGEgPSAuMjUpICsKICBsYWJzKHggPSAiTnVtYmVyIG9mIHByYWN0aWNlIHNlc3Npb25zIiwgeSA9ICJDaGFuZ2UgaW4gdGVzdCBzY29yZSIsIGNvbG91ciA9ICJUb3BpYyIsIGNhcHRpb24gPSAiRGF0YSBmcm9tIG5vb3JkZXJwb29ydC5tZW1vcnlsYWIuYXBwIikgKwogIHNjYWxlX2NvbG91cl92aXJpZGlzX2QoKSArCiAgdGhlbWVfbWwoKQpgYGAKCkl0IGxvb2tzIGxpa2UgdGhlcmUgbWlnaHQgYmUgYSBwb3NpdGl2ZSBlZmZlY3Qgb2YgcHJhY3RpY2UuCkxldCdzIGxvb2sgYXQgaXQgbW9yZSBzaW1wbHk6CklzIHRoZXJlIGEgcmVsYXRpb24gYmV0d2VlbiBzY29yZSBjaGFuZ2UgYW5kIHdoZXRoZXIgb3Igbm90IHRoZSBzdHVkZW50IGhhcyBwcmFjdGljZWQgdGhlIHRvcGljIGF0IGFsbD8KYGBge3J9Cm5wX3Njb3Jlc19hbmRfcHJhY3RpY2VbLCBkaWRfcHJhY3RpY2UgOj0gbl9yZXNwb25zZXMgPiAwXQoKYXZnX3Njb3JlX2NoYW5nZSA8LSBucF9zY29yZXNfYW5kX3ByYWN0aWNlWywgLihtZWFuX3Njb3JlX3Rlc3RfY2hhbmdlID0gbWVhbihzY29yZV90ZXN0X2NoYW5nZSkpLCBieSA9IC4odG9waWMsIHRvcGljX2xhYmVsLCBkaWRfcHJhY3RpY2UpXQoKZ2dwbG90KG5wX3Njb3Jlc19hbmRfcHJhY3RpY2UsIGFlcyh4ID0gZGlkX3ByYWN0aWNlLCB5ID0gc2NvcmVfdGVzdF9jaGFuZ2UpKSArCiAgZmFjZXRfd3JhcCh+IHRvcGljKSArCiAgZ2VvbV9obGluZSh5aW50ZXJjZXB0ID0gMCwgbGluZXR5cGUgPSAyKSArCiAgZ2VvbV92aW9saW4oZmlsbCA9ICJtaWRuaWdodGJsdWUiLCB3aWR0aCA9IC4yNSwgYWxwaGEgPSAuOCkgKwogIGdlb21fcG9pbnQoZGF0YSA9IGF2Z19zY29yZV9jaGFuZ2UsIGFlcyh5ID0gbWVhbl9zY29yZV90ZXN0X2NoYW5nZSksIHNpemUgPSAyLCBmaWxsID0gIndoaXRlIiwgcGNoID0gMjEpICsKICBnZW9tX2xhYmVsKGRhdGEgPSBhdmdfc2NvcmVfY2hhbmdlLCBhZXMoeSA9IG1lYW5fc2NvcmVfdGVzdF9jaGFuZ2UsIGxhYmVsID0gcm91bmQobWVhbl9zY29yZV90ZXN0X2NoYW5nZSwgMikpLCBudWRnZV94ID0gLjMzKSArCiAgc2NhbGVfeF9kaXNjcmV0ZShsYWJlbHMgPSBjKCJObyIsICJZZXMiKSkgKwogIGxhYnMoeCA9ICJEaWQgdGhlIHN0dWRlbnQgcHJhY3RpY2UgdGhlIHRvcGljPyIsIHkgPSAiQ2hhbmdlIGluIHRlc3Qgc2NvcmUiLCBjb2xvdXIgPSAiVG9waWMiLCBjYXB0aW9uID0gIkRhdGEgZnJvbSBub29yZGVycG9vcnQubWVtb3J5bGFiLmFwcCIpICsKICB0aGVtZV9tbCgpCgpgYGAKCkF2ZXJhZ2UgY2hhbmdlOgpgYGB7cn0KbnBfc2NvcmVzX2FuZF9wcmFjdGljZVssIC4oTiA9IC5OLCBtZWFuX3Njb3JlX3Rlc3RfY2hhbmdlID0gbWVhbihzY29yZV90ZXN0X2NoYW5nZSksIHNkX3Njb3JlX3Rlc3RfY2hhbmdlID0gc2Qoc2NvcmVfdGVzdF9jaGFuZ2UpKSwgYnkgPSAuKHRvcGljLCB0b3BpY19sYWJlbCwgZGlkX3ByYWN0aWNlKV0KYGBgCgpJcyB0aGVyZSBhIHNpZ25pZmljYW50IGRpZmZlcmVuY2UgaW4gc2NvcmUgY2hhbmdlIGJldHdlZW4gc3R1ZGVudHMgd2hvIHByYWN0aWNlZCBhbmQgdGhvc2Ugd2hvIGRpZG4ndCwgdGFraW5nIGludG8gYWNjb3VudCBkaWZmZXJlbmNlcyBpbiBzY29yZSBvbiB0aGUgZmlyc3QgdGVzdD8KYGBge3J9CmxpYnJhcnkobG1lclRlc3QpCgpsbWVyKHNjb3JlX3Rlc3RfY2hhbmdlIH4gZGlkX3ByYWN0aWNlKnNjb3JlX3Rlc3RfMSArICgxIHwgdXNlcl9pZCksIGRhdGEgPSBucF9zY29yZXNfYW5kX3ByYWN0aWNlKSB8PgogIHN1bW1hcnkoKQpgYGAKWWVzOiBwcmFjdGljaW5nIGlzIGFzc29jaWF0ZWQgd2l0aCBhbiBpbmNyZWFzZSBpbiBzY29yZSBjaGFuZ2Ugb2YgLjk2OyBzY29yaW5nIGEgcG9pbnQgaGlnaGVyIG9uIHRlc3QgMSBpcyBhc3NvY2lhdGVkIHdpdGggYSBsb3dlciBzY29yZSBjaGFuZ2UgKC0uMzApLgoKYGBge3J9CmxtZXIoc2NvcmVfdGVzdF9jaGFuZ2UgfiBkaWRfcHJhY3RpY2UqdG9waWMgKyAoMSB8IHVzZXJfaWQpLCBkYXRhID0gbnBfc2NvcmVzX2FuZF9wcmFjdGljZSkgfD4KICBzdW1tYXJ5KCkKYGBgCgoKCgpPbiBzb21lIHRvcGljcywgcGVyZm9ybWFuY2Ugb24gdGhlIHByZXRlc3Qgd2FzIGFscmVhZHkgcmVhbGx5IGhpZ2gsIGluIHdoaWNoIGNhc2Ugd2Ugd291bGQgbm90IGV4cGVjdCBtdWNoIGltcHJvdmVtZW50IGZyb20gcHJhY3RpY2UuCkxldCdzIGxvb2sgYXQgdGhlIHJlbGF0aW9uIGJldHdlZW4gcHJldGVzdCBzY29yZSBhbmQgc2NvcmUgY2hhbmdlLCB0YWtpbmcgaW50byBhY2NvdW50IHdoZXRoZXIgdGhlIHN0dWRlbnQgcHJhY3RpY2VkIG9yIG5vdDoKYGBge3J9CmdncGxvdChucF9zY29yZXNfYW5kX3ByYWN0aWNlLCBhZXMoeCA9IHNjb3JlX3Rlc3RfMSwgeSA9IHNjb3JlX3Rlc3RfMiwgY29sb3VyID0gZGlkX3ByYWN0aWNlKSkgKwogIGZhY2V0X3dyYXAofiB0b3BpYywgbmNvbCA9IDUpICsKICBnZW9tX2FibGluZShpbnRlcmNlcHQgPSAwLCBzbG9wZSA9IDEsIGxpbmV0eXBlID0gMikgKwogIGdlb21fc21vb3RoKG1ldGhvZCA9ICJsbSIpICsKICBnZW9tX3BvaW50KGFscGhhID0gLjI1KSArCiAgbGFicyh4ID0gIlByZXRlc3Qgc2NvcmUiLCB5ID0gIlBvc3R0ZXN0IHNjb3JlIiwgY29sb3VyID0gIkRpZCB0aGUgc3R1ZGVudFxucHJhY3RpY2UgdGhlIHRvcGljPyIsIGNhcHRpb24gPSAiRGF0YSBmcm9tIG5vb3JkZXJwb29ydC5tZW1vcnlsYWIuYXBwIikgKwogIHRoZW1lX21sKCkgKwogIGNvb3JkX2ZpeGVkKHhsaW0gPSBjKDAsIDQpLCB5bGltID0gYygwLCA0KSkKYGBgCgoKTGV0J3MgbG9vayBhdCBwZXJmb3JtYW5jZSBkdXJpbmcgcHJhY3RpY2UuCkFjY3VyYWN5IGJ5IHRvcGljOgpgYGB7cn0KZ2dwbG90KG5wX3ByYWN0aWNlX3N0YXRzLCBhZXMoeCA9IGFzLmNoYXJhY3Rlcih0b3BpYyksIHkgPSBhY2N1cmFjeSwgZmlsbCA9IHRvcGljKSkgKwogIGdlb21fYm94cGxvdCgpICsKICBnZW9tX2ppdHRlcih3aWR0aCA9IC4xLCBoZWlnaHQgPSAwLCBhbHBoYSA9IC41KSArCiAgbGFicyh4ID0gIlRvcGljIiwgeSA9ICJBY2N1cmFjeSIsIGZpbGwgPSAiVG9waWMiKSArCiAgc2NhbGVfeV9jb250aW51b3VzKGxpbWl0cyA9IGMoLjQsIDEpLCBsYWJlbHMgPSBzY2FsZXM6OnBlcmNlbnQpICsKICBzY2FsZV9maWxsX3ZpcmlkaXNfZCgpICsKICBndWlkZXMoZmlsbCA9ICJub25lIikgKwogIHRoZW1lX21sKCkgKwogIHRoZW1lKHBhbmVsLmdyaWQubWFqb3IueSA9IGVsZW1lbnRfbGluZShjb2xvdXIgPSAiZ3JleSIpKQpgYGAKCkFjY3VyYWN5IGJ5IHVzZXI6CmBgYHtyfQpucF9wcmFjdGljZV9zdGF0c1tuX3Jlc3BvbnNlcyA+IDEwLCAuKG1lYW5fYWNjdXJhY3kgPSBtZWFuKGFjY3VyYWN5KSwgc2RfYWNjdXJhY3kgPSBzZChhY2N1cmFjeSkpLCBieSA9IC4odXNlcl9pZCldCgpnZ3Bsb3QobnBfcHJhY3RpY2Vfc3RhdHNbbl9yZXNwb25zZXMgPiAxMF0sIGFlcyh4ID0gcmVvcmRlcihhcy5jaGFyYWN0ZXIodXNlcl9pZCksIGFjY3VyYWN5KSwgeSA9IGFjY3VyYWN5KSkgKwogIGdlb21fYm94cGxvdChvdXRsaWVyLnNoYXBlID0gTkEpICsKICBnZW9tX2ppdHRlcihhZXMoY29sb3VyID0gYXMuY2hhcmFjdGVyKHRvcGljKSksIHdpZHRoID0gLjEsIGhlaWdodCA9IDAsIGFscGhhID0gLjI1KSArCiAgbGFicyh4ID0gIlN0dWRlbnQiLCB5ID0gIkFjY3VyYXRlc3NlIiwgY29sb3VyID0gIk9uZGVyd2VycCIpICsKICBzY2FsZV95X2NvbnRpbnVvdXMobGltaXRzID0gYyguNCwgMSksIGxhYmVscyA9IHNjYWxlczo6cGVyY2VudCkgKwogIHNjYWxlX2NvbG91cl92aXJpZGlzX2QoKSArCiAgZ3VpZGVzKGZpbGwgPSAibm9uZSIpICsKICB0aGVtZV9tbCgpICsKICB0aGVtZShwYW5lbC5ncmlkLm1ham9yLnkgPSBlbGVtZW50X2xpbmUoY29sb3VyID0gImdyZXkiKSwKICAgICAgICBheGlzLnRleHQueCA9IGVsZW1lbnRfYmxhbmsoKSwKICAgICAgICBheGlzLnRpY2tzLnggPSBlbGVtZW50X2JsYW5rKCkpCmBgYAoKU3BlZWQgb2YgZm9yZ2V0dGluZyBieSBmYWN0IGFuZCB0b3BpYzoKYGBge3J9Cm5wX2ZhY3RfaWRzIDwtIHVuaXF1ZShucF9yZXNwb25zZXMkZmFjdF9pZCkKbnBfZmFjdHMgPC0gcXVlcnlfZGIocGFzdGUwKCJTRUxFQ1QgaWQgQVMgZmFjdF9pZCwgdGV4dCBGUk9NIGZhY3QgV0hFUkUgaWQgSU4gKCIsIHBhc3RlKG5wX2ZhY3RfaWRzLCBjb2xsYXBzZSA9ICIsICIpLCAiKSIpLCBkYXRhYmFzZSA9ICJzc2FhcyIpCm5wX2ZhY3RzWywgdGV4dCA6PSBnc3ViKCJcXCsiLCAiICIsIHRleHQpXQpucF9mYWN0c1ssIHRleHQgOj0gVVJMZGVjb2RlKHRleHQpXQpucF9mYWN0c1ssIHRleHQgOj0gZ3N1YigiXG4iLCAiICIsIHRleHQsIGZpeGVkID0gVFJVRSldCm5wX2ZhY3RzWywgdGV4dCA6PSBnc3ViKCLvvIsiLCAiKyIsIHRleHQsIGZpeGVkID0gVFJVRSldCgpucF9yZXNwb25zZXMgPC0gbWVyZ2UobnBfcmVzcG9uc2VzLCBucF9mYWN0cywgYnkgPSAiZmFjdF9pZCIpCgpucF9zb2YgPC0gbnBfcmVzcG9uc2VzWywgLihmaW5hbF9hbHBoYSA9IGFscGhhW3doaWNoLm1heChwcmVzZW50YXRpb25fc3RhcnRfdGltZSldKSwgYnkgPSAuKHRleHQsIHRvcGljLCB1c2VyX2lkKV0KbnBfc29mX2F2ZyA8LSBucF9zb2ZbLCAuKE4gPSAuTiwgc29mX21lYW4gPSBtZWFuKGZpbmFsX2FscGhhKSwgc29mX3NlID0gc2QoZmluYWxfYWxwaGEpL3NxcnQoLk4pKSwgYnkgPSAuKHRleHQsIHRvcGljKV0KYGBgCgpgYGB7ciBmaWcuaGVpZ2h0ID0gMTUsIGZpZy53aWR0aCA9IDh9CmdncGxvdChucF9zb2ZfYXZnW04gPiAxMF0sIGFlcyh4ID0gc29mX21lYW4sIHkgPSB0aWR5dGV4dDo6cmVvcmRlcl93aXRoaW4odGV4dCwgc29mX21lYW4sIGFzLmNoYXJhY3Rlcih0b3BpYykpLCBhbHBoYSA9IE4pKSArCiAgZmFjZXRfZ3JpZChhcy5jaGFyYWN0ZXIodG9waWMpIH4gLiwgc2NhbGVzID0gImZyZWVfeSIpICsKICBnZW9tX2Vycm9yYmFyaChhZXMoeG1pbiA9IHNvZl9tZWFuIC0gc29mX3NlLCB4bWF4ID0gc29mX21lYW4gKyBzb2Zfc2UpLCBoZWlnaHQgPSAwLCBjb2xvdXIgPSBjb2xvdXJzX21lbW9yeWxhYls1XSkgKwogIGdlb21fcG9pbnQoY29sb3VyID0gY29sb3Vyc19tZW1vcnlsYWJbNV0pICsKICBsYWJzKHkgPSAiRmVpdCIsIHggPSAiVmVyZ2VldHNuZWxoZWlkIChob2dlciA9IG1vZWlsaWprZXIpIiwgYWxwaGEgPSAiR2VvZWZlbmQgZG9vclxuYWFudGFsIHN0dWRlbnRlbiIpICsKICB0aGVtZV9tbCgpICsKICBzY2FsZV94X2NvbnRpbnVvdXMobGltaXRzID0gYyguMSwgLjUpKSArCiAgdGlkeXRleHQ6OnNjYWxlX3lfcmVvcmRlcmVkKCkgKwogIHRoZW1lKGF4aXMudGV4dC55ID0gZWxlbWVudF90ZXh0KHNpemUgPSA0KSwKICAgICAgICBwYW5lbC5ncmlkLm1ham9yLnggPSBlbGVtZW50X2xpbmUoY29sb3VyID0gImdyZXk5MCIpKQoKZ2dzYXZlKGhlcmUoIm91dHB1dCIsICJzb2ZfYnlfZmFjdF9hbmRfdG9waWMucG5nIiksIGhlaWdodCA9IDE1LCB3aWR0aCA9IDgpCmBgYAoKCgoKIyBDb25jbHVzaWVzIE5vb3JkZXJwb29ydAoKU3R1ZGVudGVuIHNjb3JlbiBnZW1pZGRlbGQgbGFnZXIgb3AgZGUgcG9zdHRlc3QgZGFuIG9wIGRlIG51bG1ldGluZy4KCkVyIHppam4gZWVuIGFhbnRhbCBmYWN0b3JlbiBkaWUgZWVuIHJvbCBzcGVsZW46CgogICAtIFN0dWRlbnRlbiBoZWJiZW4gbmlldCB6byB2ZWVsIGdlb2VmZW5kLgogICAtIERlIG9lZmVuaW5nIGRpZSB3ZWwgZ2ViZXVyZGUgdm9uZCBvdmVyIGhldCBhbGdlbWVlbiB2cmlqIHZlciB2YW4gZGUgcG9zdHRlc3QgcGxhYXRzLgogICAtIEhldCBzdGFydG5pdmVhdSB2YW4gZGV6ZSBzdHVkZW50ZW4gd2FzIGFsIHZyaWogaG9vZywgd2FhcmRvb3IgZXIgbWluZGVyIHJ1aW10ZSB2b29yIHZlcmJldGVyaW5nIHdhcywgZW4gZXh0cmEgb2VmZW5pbmcgaW4gZGV6ZSB2b3JtIHdlbGxpY2h0IG5pZXQgem8gemludm9sIHdhcy4KICAgLSBBbHMgd2Uga2lqa2VuIG5hYXIgZGUgY29tYmluYXRpZSB2YW4gdGVzdHNjb3JlcyBlbiBvZWZlbmFjdGl2aXRlaXQgb3AgaW5kaXZpZHVlbGUgb25kZXJkZWxlbiwgemllbiB3ZSBlZW4gYWFudGFsIHBhdHJvbmVuOgogICAgLSBWZWVsIG9lZmVuaW5nIGdlYmV1cmRlIG9wIG9uZGVyZGVsZW4gd2FhciBoZXQgc3RhcnRuaXZlYXUgYWwgaG9vZyB3YXMsIHpvYWxzIEFmcm9uZGVuLCBBZnRyZWtrZW4gJiBPcHRlbGxlbiwgCgoKLS0tCgoKIyBBbGZhIGNvbGxlZ2UKCmBgYHtyfQphbGZhX2RvbWFpbiA8LSBxdWVyeV9kYigiU0VMRUNUICogRlJPTSBkb21haW4gV0hFUkUgbmFtZSA9ICdhbGZhLm1lbW9yeWxhYi5hcHAnOyIsIGRhdGFiYXNlID0gInNsaW1zdGFtcGVuIikKYGBgCgpVc2VycyByZWdpc3RlcmVkIG9uIHRoaXMgZG9tYWluOgpgYGB7cn0KYWxmYV91c2VycyA8LSBxdWVyeV9kYihwYXN0ZTAoIlNFTEVDVCBpZCBBUyB1c2VyX2lkIEZST00gdXNlcnMgV0hFUkUgZG9tYWluX2lkID0gIiwgYWxmYV9kb21haW4kaWQsICI7IiksIGRhdGFiYXNlID0gInNsaW1zdGFtcGVuIikKYGBgCgpMZXNzb25zIG9uIHRoaXMgZG9tYWluOgpgYGB7cn0KYWxmYV9sZXNzb25zIDwtIHF1ZXJ5X2RiKHBhc3RlMCgiU0VMRUNUICogRlJPTSBsZXNzb24gV0hFUkUgZG9tYWluX2lkID0gIiwgYWxmYV9kb21haW4kaWQsICI7IiksIGRhdGFiYXNlID0gInNsaW1zdGFtcGVuIikKYGBgCgpTZXNzaW9ucyBmcm9tIHVzZXJzIG9uIHRoaXMgZG9tYWluIGR1cmluZyB0aGUgcGlsb3QgcGVyaW9kOgpgYGB7cn0KYWxmYV9zZXNzaW9ucyA8LSBxdWVyeV9kYihwYXN0ZTAoIlNFTEVDVCAqIEZST00gc2Vzc2lvbiBXSEVSRSB0b2tlbl9pZCA9IDIgQU5EIHVzZXJfaWQgSU4gKCIsIHBhc3RlKGFsZmFfdXNlcnMkdXNlcl9pZCwgY29sbGFwc2UgPSAiLCAiKSwgIik7IiksIGRhdGFiYXNlID0gInNzYWFzIikKYWxmYV9zZXNzaW9ucyA8LSBhbGZhX3Nlc3Npb25zW2NyZWF0ZV90aW1lID4gIjIwMjQtMTEtMDEiXQpgYGAKCldoZW4gd2VyZSB0aGVzZSBzZXNzaW9ucz8KYGBge3J9CmFsZmFfc2Vzc2lvbnNbLCBzZXNzaW9uX2RhdGUgOj0gYXMuRGF0ZShjcmVhdGVfdGltZSldCgpnZ3Bsb3QoYWxmYV9zZXNzaW9ucywgYWVzKHggPSBzZXNzaW9uX2RhdGUpKSArCiAgZ2VvbV9oaXN0b2dyYW0oYmlud2lkdGggPSAxLCBmaWxsID0gY29sb3Vyc19tZW1vcnlsYWJbMV0pICsKICBsYWJzKHggPSAiRGF0ZSIsIHkgPSAiU2Vzc2lvbnMgcGVyIGRheSIsIGNhcHRpb24gPSAiRGF0YSBmcm9tIGFsZmEubWVtb3J5bGFiLmFwcCIpICsKICBzY2FsZV95X2NvbnRpbnVvdXMoZXhwYW5kID0gYygwLCAwKSkgKwogIHRoZW1lX21sKCkgKwogIHRoZW1lKHBhbmVsLmdyaWQubWFqb3IueSA9IGVsZW1lbnRfbGluZShjb2xvdXIgPSAiZ3JleTkwIikpCmBgYAoKTW9zdCBwb3B1bGFyIGRheXM6CmBgYHtyfQphbGZhX3Nlc3Npb25zWywgLk4sIGJ5ID0gLihzZXNzaW9uX2RhdGUpXVtvcmRlcigtTildCmBgYApUb3RhbCBzZXNzaW9uczoKYGBge3J9Cm5yb3coYWxmYV9zZXNzaW9ucykKYGBgCgpTZXNzaW9ucyBwZXIgdXNlcjoKYGBge3J9CmFsZmFfc2Vzc2lvbnNbLCAuTiwgYnkgPSB1c2VyX2lkXVtvcmRlcigtTildCmBgYAoKVG90YWwgdXNlcnMgd2l0aCBhdCBsZWFzdCBvbmUgc2Vzc2lvbjoKYGBge3J9Cmxlbmd0aCh1bmlxdWUoYWxmYV9zZXNzaW9ucyR1c2VyX2lkKSkKYGBgCgoKV2hpY2ggbGVzc29ucyBkaWQgdXNlcnMgZG8/IFBhcnNlIHRoZSBzZXNzaW9uIGNvbnRleHQ6CmBgYHtyfQphbGZhX3Nlc3Npb25zX3BhcnNlZCA8LSBhbGZhX3Nlc3Npb25zWywgbWFwX2Rmcihjb250ZXh0LCBmcm9tSlNPTildIHw+IHNldERUKCkKYGBgCgpgYGB7cn0KYWxmYV9zZXNzaW9uc19wYXJzZWRbLCAuKGBLZWVyIGdlb2VmZW5kYCA9IC5OKSwgYnkgPSAuKExlcyA9IHRpdGxlKV1bb3JkZXIoLWBLZWVyIGdlb2VmZW5kYCldIHw+IGtuaXRyOjprYWJsZSgpCmBgYAoKTWFzdGVyeSBjcmVkaXRzOgpgYGB7cn0KYWxmYV9jcmVkaXRzIDwtIHF1ZXJ5X2RiKHBhc3RlMCgiU0VMRUNUICogRlJPTSBsZXNzb25fbWFzdGVyZWQgV0hFUkUgdXNlcl9pZCBJTiAoIiwgcGFzdGUoYWxmYV91c2VycyR1c2VyX2lkLCBjb2xsYXBzZSA9ICIsICIpLCAiKTsiKSwgZGF0YWJhc2UgPSAic2xpbXN0YW1wZW4iKQpgYGAKCkFkZCBsZXNzb24gdGl0bGVzOgpgYGB7cn0KYWxmYV9jcmVkaXRzIDwtIG1lcmdlKGFsZmFfY3JlZGl0cywgYWxmYV9sZXNzb25zWywgLihsZXNzb25faWQgPSBpZCwgdGl0bGUpXSkKYGBgCgpgYGB7cn0KYWxmYV9jcmVkaXRzWywgLk4sIGJ5ID0gLih0aXRsZSldW29yZGVyKC1OKV0KYGBgCgoKCgpgYGB7cn0KcXVlcnlfZGIocXVlcnkgPSAiU0VMRUNUICoKRlJPTSBwZ19jYXRhbG9nLnBnX3RhYmxlcwpXSEVSRSBzY2hlbWFuYW1lICE9ICdwZ19jYXRhbG9nJyBBTkQgCiAgICBzY2hlbWFuYW1lICE9ICdpbmZvcm1hdGlvbl9zY2hlbWEnOyIsCiAgICAgICAgIGRhdGFiYXNlID0gInNsaW1zdGFtcGVuIikKYGBgCgo=